;+
; NAME:
;       READ_BGPS_CSV
;
; PURPOSE:
;       Read in the BGPS Distance Database CSV file into a structure
;       for use with other routines.
;
; CATEGORY:
;       BGPS distance-omnibus Utility
;
; CALLING SEQUENCE:
;       struct_arr = read_bgps_csv( csv_filename, csv_struct
;                                   [,CSV=csv_struct] [,/VERBOSE] )
;
; INPUTS:
;       csv_filename -- Filename of the latest BGPS Distance Database
;                       CSV file [Default: bgps_distance_database.csv]
;
; OPTIONAL INPUTS:
;       NULL    -- String with which to replace 'null' in input CSV
;                  (default: '')
;
; KEYWORD PARAMETERS:
;       VERBOSE -- Print out information about the input CSV file
;
; OUTPUTS:
;       struct_arr -- An array of structures (ncols long), each
;                     containing the information from a single row of
;                     the input CSV file.
;       csv_struct -- The structure containing information about the
;                     CSV file, including size and column headers
;
; OPTIONAL OUTPUTS:
;       csv_struct -- The structure containing information about the
;                     CSV file, including size and column headers
;
; EXAMPLE:
;       struct_arr = read_bgps_csv('bgps_distance_database.csv',
;                                  csv_struct)
;            -- OR --
;       s = read_bgps_csv( CSV=csv_struct, /VER )
;
; NOTES:
;       This routine is a modified version of the routine READ_CSV.pro
;       introduced in IDL v7.1.
;
; MODIFICATION HISTORY:
;       Created:  05/28/10, TPEB -- Initial version is heavily modified
;                                   version of READ_CSV.pro (IDL 7.1)
;       Modified: 06/03/10, TPEB -- Changed output from a structure of
;                                   arrays to an array of structures
;                                   (each structure contains info on a
;                                   single BGPS source).
;       Modified: 08/25/11, TPEB -- Made input of filename optional,
;                                   since I only use one all the
;                                   time.  Conformal changes.
;       Modified: 11/30/11, TPEB -- Documentation cleanup.
;       Modified: 03/18/13, TPEB -- Noted change in location of the
;                                   old CSV.
;-

FUNCTION READ_BGPS_CSV, filename, csv_struct, CSV=csv_str, $
                        NULL=nullstr, VERBOSE=verbose
  
  ;; Default: If no filename specified, use
  ;; 'bgps_distance_database.csv'
  IF n_elements(filename) EQ 0 THEN $
     filename = './obsolete/bgps_distance_database.csv'
  
     
  ;; Determine the size of the CSV file -- there are 3 header lines!
  nline = file_lines(filename)
  nrows = nline - 3
  
  ;; Set the 'null' replacement
  IF n_elements(nullstr) EQ 0 THEN nullstr=''
  
  ;; Open the CSV file
  openr, lun, filename, /get_lun

  ;; Get the 1st line (Column Names) & parse into header_vars
  header = string('')
  readf, lun, header
  header_vars = strsplit(header, ',', /EXTRACT, COUNT=ncols)
  IF ncols EQ 0 THEN message,'Header contains no columns'
  header_vars[0] = (stregex(header_vars[0],'(# *)?(.*)',/SUBEXP,/EXTRACT))[2]
  
  ;; Get the 2nd line (Data Types) & Parse
  datatype = string('')
  readf, lun, datatype
  type_vars = strsplit(datatype, ',', /EXTRACT, COUNT=ncols2)
  IF ncols2 NE ncols THEN message,'Mismatch between column names and data types'
  type_vars[0] = (stregex(type_vars[0],'(# *)?(.*)',/SUBEXP,/EXTRACT))[2]

  ;; Get the 3rd line (units) & Parse
  units = string('')
  readf, lun, units
  unit_vars = strsplit(units, ',', /EXTRACT, COUNT=ncols3)
  IF ncols3 NE ncols THEN message,'Mismatch between column names and units'
  unit_vars[0] = (stregex(unit_vars[0],'(# *)?(.*)',/SUBEXP,/EXTRACT))[2]
  
  ;; Build array of data type flags for structure building  (with
  ;; error checking)
  objidp = bytarr(ncols)
  types = strtrim(type_vars,2)
  FOR i=0L, ncols-1 DO BEGIN
     IF types[i] EQ 'double' THEN objidp[i] = 0 ELSE $
        IF types[i] EQ 'int'    THEN objidp[i] = 1 ELSE $
           IF types[i] EQ 'char'   THEN objidp[i] = 2 $
           ELSE message,$
        string(type_vars[i],header_vars[i],format=$
               "('Data type ',A0,' for column ',A0,' is invalid!')")
  ENDFOR

  ;; Create Structure for single BGPS source
  CASE objidp[0] OF
     0 : outstruct = CREATE_STRUCT(header_vars[0], double(0))
     1 : outstruct = CREATE_STRUCT(header_vars[0], long(0))
     2 : outstruct = CREATE_STRUCT(header_vars[0], '')
  ENDCASE
  
  FOR i=1L, ncols-1 DO BEGIN
     CASE objidp[i] OF
        0 : outstruct = CREATE_STRUCT(outstruct, header_vars[i], double(0))
        1 : outstruct = CREATE_STRUCT(outstruct, header_vars[i], long(0))
        2 : outstruct = CREATE_STRUCT(outstruct, header_vars[i], '')
     ENDCASE
  ENDFOR
  
  ;; Replicate structure into an array of structures
  outstruct = REPLICATE(outstruct, nrows)

  
  ;; Create Structure to hold information about the CSV file itself
  ;; Add structure elements for table dimensions
  csv_struct = CREATE_STRUCT('nrows', nrows, 'ncols', ncols)
  ;; Add structure elements for the header rows (as strarr)
  csv_struct = CREATE_STRUCT(csv_struct, 'col_name', header_vars, $
                            'col_type', type_vars, 'col_units', unit_vars)


  ;; Read in the CSV file!
  FOR i=0L, nrows-1 DO BEGIN
     input_line = string('')
     readf, lun, input_line
     values = strsplit(input_line, ',', /EXTRACT, COUNT=nlinecols)
     
     ;; Error Checking
     IF nlinecols NE ncols THEN message,$
        'Incorrect number of columns in line ' + string(i+1,format='(I0)')
     
     ;; 'null' replacement
     wherenull = where(values EQ 'null', nnull)
     IF nnull GT 0 THEN values[wherenull] = nullstr
     
     ;; Fill the structure
     FOR j=0, ncols-1 DO BEGIN
        CASE objidp[j] OF
           0 : outstruct[i].(j) = double(values[j])
           1 : outstruct[i].(j) = long(values[j])
           2 : outstruct[i].(j) = values[j]
        ENDCASE
     ENDFOR
  ENDFOR

  ;; Clean Up
  close, lun
  free_lun, lun
  
  ;; Print out size information, if /VERBOSE
  IF KEYWORD_SET( verbose ) THEN $
     message,$
     string(ncols, nrows, format=$
            "('Read in CSV file with ',I0,' columns and ',I0,' rows.')"),/inf
  
  ;; Return the CSV struct in the keyword
  csv_str = csv_struct
  
  RETURN, outstruct
  
END
